from wudi_crawler.crawler import Crawler
import asyncio


async def fetch(url):
    # async with Crawler(headless=False, selectors={'time_selectors': ['.nav01 h6 span'], 'content_selectors': [".news-content, .news-edit-info"], 'nav_selectors': [], 'author_selectors': []}) as crawler:
    async with Crawler(headless=False, selectors={"content_selectors": ["#text"]}) as crawler:
        result = await crawler.fetch(url, return_fields=["title", "content", "markdown", "byline"])
        print(result["markdown"])
        with open("a.html", encoding="utf-8", mode="w") as f:
            f.write(result["content"])
            f.write(result["byline"])


url = "https://www.lib.whu.edu.cn/engine2/d/23883829/5619257/0?t=11166596&p=1"
url = "https://jstgw.nju.edu.cn/info/1002/1445.htm"
url = "https://www.shobserver.com/staticsg/res/html/web/newsDetail.html?id=1008885&v=2.0&sid=11"
url = "https://baijiahao.baidu.com/s?id=1847371310991360658&wfr=spider&for=pc"
url = "http://www.wztsg.com/news/show/3859.html"
url = "https://www.gglib.org.cn/xxzx/gtdt/3736.html"
url = "https://www.shio.gov.cn/TrueCMS/shxwbgs/ywts/content/d4d3568a-f799-46db-b71f-d16d30632bcb.htm"
url = "https://www.shobserver.com/staticsg/res/html/web/newsDetail.html?id=1008885&v=2.0&sid=11"
print("start")
asyncio.run(fetch(url))
